﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Amib.Threading;
using HtmlAgilityPack;
using log4net;
using System.Reflection;
using System.Xml;
using System.Text.RegularExpressions;

using WebCollection.Tags;
using System.IO;
using System.Threading;
using Util;
using DAL;

namespace WebCollection
{
    public class Robot : IDisposable
    {
        private WebUtil webUtil = new WebUtil();

        private static ILog log = log4net.LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);

        private Stack<string> queUrl = new Stack<string>();

        private SmartThreadPool SMT = new SmartThreadPool(20, 1, 1);

        ImgDownloader imgdownloader = new ImgDownloader();

        private UrlFilter urlFilter = new UrlFilter();

        private Dir dir = null;

        private GatherRule Grule = null;

        private CsvDb csvDb = null;

        public void Close() 
        {
            if (csvDb != null) 
            {
                csvDb.Close();
            }
        }

        public Robot()
        {
        }

        internal Robot(GatherRule grule, string savepath)
        {
            this.Grule = grule;
            dir = new Dir(savepath);
        }

        public Robot(string configpath, string outputpath,bool enablerepeation)
        {
            Grule = new GatherRule(configpath);
            dir = new Dir(outputpath);
            csvDb = new CsvDb(outputpath, enablerepeation);
            log.Info("task begin：" + dir.root);
        }

        public void Start(String seed)
        {
            if (string.IsNullOrEmpty(seed))
            {
                log.Info("seed url should not be empty");
                return;
            }

            try
            {
                queUrl.Push(seed);
                while (queUrl.Count > 0)
                {
                    string url = queUrl.Pop();

                    try
                    {
                        TPage tpage = Grule.Match(url);
                        AnalyzeTPage(url, tpage);
                    }
                    catch (Exception ex)
                    {
                        log.Warn(ex.Message);
                        log.Warn(ex.StackTrace);
                    }
                }
                SMT.WaitForIdle();
            }
            catch (Exception e)
            {
                log.Info(e.Message);
                log.Info(e.StackTrace);
            }
        }

        private void AnalyzeTPage(string url, TPage tpage)
        {
            log.Info("pop :" + url);

            if (tpage.Des == "non-index")
            {
                HandleWithTSave(url, tpage);
            }
            else if (tpage.Des == "index")
            {
                DealIndexPage(url, tpage);
            }
        }

        

        private void DealIndexPage(string url, TPage tpage)
        {
            HtmlDocument doc = webUtil.DownDomHtml(url, tpage.Dynamic);

            bool repeation=HandleWithTSave_M(url, tpage, doc);
            if (repeation) 
                return;
            HandleWithTEnqueue(url, tpage, doc);
        }

        private bool HandleWithTSave_M(string url, TPage tpage, HtmlDocument doc)
        {
            TSave_M tsave_m = tpage.TSave_M;
            if (tsave_m == null) return false ;

            string xpathroot = tsave_m.RootxPath;
            List<TField> fields = tsave_m.TFields;

            string classname = tsave_m.ClassName;

            HtmlNodeCollection hnc = doc.DocumentNode.SelectNodes(xpathroot);
            if (hnc == null) throw new Exception("positioning failed \r\n" + xpathroot + "\r\n" + url);

            //构造XML
            XmlDocument xdResult = new XmlDocument();
            XmlDeclaration dec = xdResult.CreateXmlDeclaration("1.0", "UTF-8", null);
            xdResult.AppendChild(dec);
            XmlElement root = xdResult.CreateElement("Product");

            XmlElement xe_classname_s = xdResult.CreateElement(classname + "s");
            root.AppendChild(xe_classname_s);

            xdResult.AppendChild(root);

            for (int i = 0; i < hnc.Count(); i++)
            {
                HtmlNode hn = hnc[i];
                AddClass(url, fields, hn, classname, xe_classname_s, xdResult, i);
            }
            //保存多次
            string filename = dir.GetFileName(url);
            xdResult.Save(filename);

            string savepath = filename.Replace(".xml", "");
            if (!Directory.Exists(savepath))
            {
                Directory.CreateDirectory(savepath);
            }
            AnalyzeXdResult(xdResult, savepath);

            //判重并写入csv
            Entity entity = new Entity() { xmlFile = filename, imgDir = savepath };
            bool repeation= csvDb.Add(entity);
            return repeation;
        }

        private void AddClass(string url, List<TField> fields, HtmlNode hn, string classname, XmlElement root, XmlDocument xdResult, int identity)
        {
            //构造classname
            XmlElement xeClassName = xdResult.CreateElement(classname);
            //xe_1.InnerText = value_c;
            root.AppendChild(xeClassName);


            XmlElement xe_i = xdResult.CreateElement("id");
            xe_i.InnerText = identity.ToString();
            xeClassName.AppendChild(xe_i);



            for (int i = 0; i < fields.Count(); i++)
            {
                TField tfield = fields[i];
                AnalyzeTField(tfield, hn, xeClassName, xdResult, url);
            }
        }

        private void HandleWithTSave(string url, TPage tpage)
        {
            TSave tsave = tpage.TSave;
            if (tsave == null) return;

            smtparam ufo = new smtparam(tpage.Dynamic, url, tsave);

            if (tpage.Threads)
            {
                SMT.QueueWorkItem(new WorkItemInfo { Timeout = 300000 }, ThreadMethod, ufo);
            }
            else
            {
                ThreadMethod(ufo);
            }
        }
        [STAThread]
        private object ThreadMethod(Object o)
        {
            smtparam ufo = o as smtparam;
            string url = ufo.url;
            TSave tsave = ufo.TSave;

            string xpathroot = tsave.RootxPath;
            List<TField> fields = tsave.TFields;

            HtmlDocument doc = webUtil.DownDomHtml(url, ufo.dynamic);

            HtmlNodeCollection hnc = doc.DocumentNode.SelectNodes(xpathroot);
            if (hnc == null) throw new Exception("positioning failed \r\n" + xpathroot + "\r\n" + url);


            foreach (HtmlNode hn in hnc)
            {
                SaveResult(url, fields, hn);
            }
            return null;
        }

        private void SaveResult(string url, List<TField> fields, HtmlNode hn)
        {
            XmlDocument xdResult = new XmlDocument();
            XmlDeclaration dec = xdResult.CreateXmlDeclaration("1.0", "UTF-8", null);
            xdResult.AppendChild(dec);
            XmlElement root = xdResult.CreateElement("Product");
            xdResult.AppendChild(root);

            foreach (TField tfield in fields)
            {
                AnalyzeTField(tfield, hn, root, xdResult, url);
            }

            string filename = dir.GetFileName(url);
            xdResult.Save(filename);

            string savepath = filename.Replace(".xml", "");

            AnalyzeXdResult(xdResult, savepath);

        }

        private void AnalyzeXdResult(XmlDocument xdResult, string savepath)
        {
            XmlNodeList xnlNewEntry = xdResult.SelectNodes(@"//*[@newentry]");
            if (xnlNewEntry == null || xnlNewEntry.Count == 0)
            {
                return;
            }
            Console.Write("downloading imgs ");
            foreach (XmlNode xn in xnlNewEntry)
            {
                string newentry = xn.Attributes["newentry"].Value;
                Entry entry = (Entry)Enum.Parse(typeof(Entry), newentry);
                switch (entry)
                {
                    case Entry.nonentry:
                        break;
                    case Entry.downimg:
                        string seed = xn.InnerText;
                        string identity = xn.ParentNode.SelectSingleNode("./id").InnerText;
                        string filepath1 = Path.Combine(savepath, identity + ".jpg");
                        Thread.Sleep(3000);
                        imgdownloader.AddToThreadPool(seed, filepath1);

                        break;
                    case Entry.downhtml:
                        string seed1 = xn.InnerText;
                        Robot robot1 = new Robot(this.Grule, savepath + "_" + xn.LocalName);
                        robot1.Start(seed1);
                        robot1.Dispose();
                        break;
                    default:
                        break;
                }
            }
            imgdownloader.EndAllTask();
            Console.WriteLine();
        }

        private void AnalyzeTField(TField tfield, HtmlNode hn, XmlElement root, XmlDocument xdResult, string url)
        {
            try
            {
                Entry newentry = tfield.newentry;
                string name = tfield.Name;

                TConst tconst = tfield.TConst;
                TXPath txpath = tfield.TXPath;
                TRegex tregex = tfield.TRegex;

                //固定值
                if (txpath == null)
                {
                    string value_c = string.Empty;
                    EConst ec = tconst.eConst;
                    switch (ec)
                    {
                        case EConst.url:
                            value_c = url;
                            break;
                        case EConst.now:
                            value_c = DateTime.Now.ToString();
                            break;
                        case EConst.value:
                            value_c = tconst.tValue.Value;
                            break;
                    }

                    XmlElement xe_1 = xdResult.CreateElement(name);
                    xe_1.InnerText = value_c;

                    //newentry="downimg"
                    //newentry="downhtml"
                    switch (newentry)
                    {
                        case Entry.nonentry:
                            break;
                        case Entry.downimg:
                            XmlAttribute xa_1 = xdResult.CreateAttribute("newentry");
                            xa_1.Value = "downimg";
                            xe_1.Attributes.Append(xa_1);
                            break;
                        case Entry.downhtml:
                            XmlAttribute xa_2 = xdResult.CreateAttribute("newentry");
                            xa_2.Value = "downhtml";
                            xe_1.Attributes.Append(xa_2);
                            break;
                        default:
                            break;
                    }

                    //if (newentry)
                    //{
                    //    XmlAttribute xa_1 = xdResult.CreateAttribute("newentry");
                    //    xa_1.Value = "true";
                    //    xe_1.Attributes.Append(xa_1);
                    //}
                    root.AppendChild(xe_1);
                    //log.Info(name + " : " + value_c);
                    return;
                }

                //xpath
                string value_x = AnalyzeTXPath(txpath, hn, url);


                if (tregex == null)
                {
                    XmlElement xe = xdResult.CreateElement(name);
                    xe.InnerText = value_x;

                    switch (newentry)
                    {
                        case Entry.nonentry:
                            break;
                        case Entry.downimg:
                            XmlAttribute xa_1 = xdResult.CreateAttribute("newentry");
                            xa_1.Value = "downimg";
                            xe.Attributes.Append(xa_1);
                            break;
                        case Entry.downhtml:
                            XmlAttribute xa_2 = xdResult.CreateAttribute("newentry");
                            xa_2.Value = "downhtml";
                            xe.Attributes.Append(xa_2);
                            break;
                        default:
                            break;
                    }


                    //if (newentry)
                    //{
                    //    XmlAttribute xa_1 = xdResult.CreateAttribute("newentry");
                    //    xa_1.Value = "true";
                    //    xe.Attributes.Append(xa_1);
                    //}
                    root.AppendChild(xe);
                    //log.Info(name + " : " + value_x);
                    return;
                }

                //正则格式化
                string value_rf = AnalyzeTRegex(tregex, value_x);
                XmlElement xe_rf = xdResult.CreateElement(name);
                xe_rf.InnerText = value_rf;

                switch (newentry)
                {
                    case Entry.nonentry:
                        break;
                    case Entry.downimg:
                        XmlAttribute xa_rf = xdResult.CreateAttribute("newentry");
                        xa_rf.Value = "downimg";
                        xe_rf.Attributes.Append(xa_rf);
                        break;
                    case Entry.downhtml:
                        XmlAttribute xa_rf1 = xdResult.CreateAttribute("newentry");
                        xa_rf1.Value = "downhtml";
                        xe_rf.Attributes.Append(xa_rf1);
                        break;
                    default:
                        break;
                }

                //if (newentry)
                //{
                //    XmlAttribute xa_rf = xdResult.CreateAttribute("newentry");
                //    xa_rf.Value = "true";
                //    xe_rf.Attributes.Append(xa_rf);
                //}
                root.AppendChild(xe_rf);
                //log.Info(name + " : " + value_rf);
                return;
            }
            catch (Exception e)
            {
                log.Warn(e.Message);
            }
        }

        private string AnalyzeTRegex(TRegex tregex, string value_x)
        {
            Regex regex = tregex.TPattern.Regex;

            string format = tregex.TFormat == null ? "" : tregex.TFormat.Format;

            Match m = regex.Match(value_x);

            if (string.IsNullOrEmpty(format)) //默认返回组1
            {
                return m.Groups[1].Value;
            }

            List<string> values = new List<string>();
            for (int i = 1; i < m.Groups.Count; i++)
            {
                string gv = m.Groups[i].Value;
                values.Add(gv);
            }

            string value = format.Format(values);
            return value;
        }

        private string AnalyzeTXPath(TXPath txpath, HtmlNode hnPre, string url)
        {
            string xpath = txpath.Xpath;
            string attri = txpath.Attri;
            string position = txpath.Position;

            if (xpath.StartsWith("./@"))
            {
                String tmp = hnPre.Attributes[attri].Value;
                return tmp;
            }

            HtmlNode hn = hnPre.SelectSingleNode(xpath);
            if (hn == null)
                throw new Exception("positioning failed " + xpath);

            if (!string.IsNullOrEmpty(attri))
            {
                string va = hn.Attributes[attri].Value;
                if (attri == "href" || attri == "src") //转绝对地址
                {
                    va = UrlUtil.FullUrl(url, va);
                }
                return va;
            }

            string value = Positioning(hn, position);



            return value;
        }

        private string Positioning(HtmlNode hn, string position)
        {
            switch (position)
            {
                case "innertext":
                    return hn.InnerText;
                case "src":
                    return hn.Attributes["src"].Value;
                case "href":
                    return hn.Attributes["href"].Value;
                case "outerhtml":
                    return hn.OuterHtml;
                case "innerhtml":
                    return hn.InnerHtml;
                case "onclick":
                    return hn.Attributes["onclick"].Value;
                default:
                    throw new Exception("(innerText,innerHtml,outerHtml) not include " + position);
            }
        }

        private void HandleWithTEnqueue(string url, TPage tpage, HtmlDocument doc)
        {
            TEnqueue ten = tpage.TEnqueue;
            if (ten == null) return;

            List<TXPath> list = ten.TXPaths;
            //if (list == null)
            //    throw new Exception("有enqueue无xpath");

            foreach (TXPath txp in list)
            {
                try
                {
                    AnalyzeTXPath(doc, txp, url);
                }
                catch (Exception e)
                {
                    log.Info(e.Message);
                    continue;
                }
            }
        }

        private void AnalyzeTXPath(HtmlDocument doc, TXPath txp, string url)
        {
            string xpath = txp.Xpath;
            string attri = txp.Attri;

            HtmlNodeCollection hnc = doc.DocumentNode.SelectNodes(xpath);
            if (hnc == null)
                throw new Exception("positioning failed " + xpath);
            foreach (HtmlNode hn in hnc)
            {
                string link = hn.Attributes[attri].Value;
                string full = UrlUtil.FullUrl(url, link);

                //布隆过滤
                if (urlFilter.ContainUrl(full))
                {
                    continue;
                }

                log.Info("push:" + link.Replace("http://www.dianping.com", ""));

                queUrl.Push(full);
            }
        }

        

        internal class smtparam
        {
            public bool dynamic { get; set; }
            public string url { get; set; }
            public TSave TSave { get; set; }

            public smtparam(bool dynamic, string url, TSave tsave)
            {
                this.dynamic = dynamic;
                this.url = url;
                this.TSave = tsave;
            }
        }

        public void Dispose()
        {
            if (imgdownloader != null)
            {
                imgdownloader.Dispose();
            }
            if (SMT != null)
            {
                SMT.Shutdown();
            }
        }
    }
}
